#!/bin/bash
# 内存分配器碎片过多自动处理脚本
# 适用于检测和优化系统及进程内存碎片问题
FIX_CODE_ID="fragmentation"

# 配置参数
FRAGMENTATION_THRESHOLD=30  # 内存碎片率阈值(%)，超过此值触发处理
TARGET_PROCESSES=("java" "nginx" "mysql")  # 需要重点监控的进程
LOG_FILE="/var/log/memory_fragmentation.log"
JEMALLOC_CONF="/etc/jemalloc.conf"  # jemalloc配置文件路径
RESTART_DELAY=60  # 进程重启间隔(秒)，避免同时重启多个服务
MAX_RESTART_COUNT=3  # 单日最大重启次数

# 日志函数
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" >> "$LOG_FILE"
}

# 检查系统内存碎片情况
check_system_fragmentation() {
    # 通过/proc/buddyinfo计算内存碎片率
    # 不同系统可能需要调整节点索引
    local total_free=0
    local total_fragments=0
    
    while read -r line; do
        if [[ $line == Node* ]]; then
            # 提取各阶空闲页框数量
            local counts=($(echo "$line" | awk '{for(i=5;i<=NF;i++) print $i}'))
            
            # 计算总空闲内存(以4KB页为单位)
            for i in "${!counts[@]}"; do
                local order=$i
                local pages=${counts[$i]}
                local size=$((pages * (1 << order)))  # 2^order个4KB页
                total_free=$((total_free + size))
                
                # 计算碎片贡献(小页数量占比)
                if [ $order -lt 4 ]; then  # 小于64KB的页视为碎片
                    total_fragments=$((total_fragments + size))
                fi
            done
        fi
    done < /proc/buddyinfo
    
    if [ $total_free -eq 0 ]; then
        echo 0
        return
    fi
    
    # 计算碎片率(%)
    echo $(( (total_fragments * 100) / total_free ))
}

# 检查指定进程的内存碎片情况
check_process_fragmentation() {
    local proc_name=$1
    local pids=$(pgrep -x "$proc_name")
    
    if [ -z "$pids" ]; then
        log "进程$proc_name未运行，无法检查碎片"
        return 1
    fi
    
    for pid in $pids; do
        if [ -f "/proc/$pid/smaps" ]; then
            # 计算该进程的内存碎片指标
            local anon_pages=$(grep -A 5 "AnonPages:" /proc/$pid/smaps | grep "Pss:" | awk '{sum += $2} END {print sum}')
            local heap_frag=$(grep -A 10 "Heap:" /proc/$pid/smaps | grep "Size:" | awk '{size=$2} END {print size}')
            
            if [ -n "$anon_pages" ] && [ -n "$heap_frag" ] && [ $heap_frag -gt 0 ]; then
                local frag_ratio=$(( (anon_pages * 100) / heap_frag ))
                if [ $frag_ratio -gt $FRAGMENTATION_THRESHOLD ]; then
                    log "进程$proc_name(PID:$pid)内存碎片率过高: $frag_ratio%"
                    echo "$pid $proc_name $frag_ratio"
                fi
            fi
        fi
    done
    
    return 0
}

# 检查当日重启次数
check_restart_count() {
    local today=$(date +'%Y-%m-%d')
    local count=$(grep "$today" "$LOG_FILE" | grep "成功重启进程" | wc -l)
    
    if [ $count -ge $MAX_RESTART_COUNT ]; then
        log "已达到单日最大重启次数($MAX_RESTART_COUNT)，停止自动重启"
        return 1
    fi
    return 0
}

# 重启进程以清理碎片
restart_process() {
    local pid=$1
    local proc_name=$2
    
    log "准备重启进程$proc_name(PID:$pid)以清理内存碎片"
    
    # 检查是否达到重启限制
    if ! check_restart_count; then
        return 1
    fi
    
    # 尝试优雅重启
    if systemctl is-active --quiet "$proc_name"; then
        log "通过systemctl优雅重启$proc_name"
        if systemctl restart "$proc_name"; then
            log "成功重启进程$proc_name"
            sleep $RESTART_DELAY
            return 0
        fi
    fi
    
    # 优雅重启失败则强制重启
    log "尝试强制终止进程$pid"
    if kill -9 $pid; then
        log "成功终止进程$pid"
        sleep 2
        
        # 启动进程
        if systemctl start "$proc_name"; then
            log "成功启动进程$proc_name"
            sleep $RESTART_DELAY
            return 0
        else
            log "启动进程$proc_name失败，需要人工干预"
            return 1
        fi
    else
        log "强制终止进程$pid失败"
        return 1
    fi
}

# 配置jemalloc内存分配器优化碎片
configure_jemalloc() {
    log "配置jemalloc优化内存碎片"
    
    # 检查jemalloc是否安装
    if ! command -v jemalloc-config &> /dev/null; then
        log "未安装jemalloc，尝试安装"
        if command -v apt &> /dev/null; then
            apt-get install -y jemalloc
        elif command -v yum &> /dev/null; then
            yum install -y jemalloc
        else
            log "无法安装jemalloc，不支持的包管理器"
            return 1
        fi
    fi
    
    # 配置jemalloc参数优化碎片
    if [ ! -f "$JEMALLOC_CONF" ]; then
        cat > "$JEMALLOC_CONF" << EOF
# 优化内存碎片配置
mallopt.dirty_decay_ms = 1000
mallopt.muzzy_decay_ms = 1000
mallopt.max_background_threads = 4
mallopt.tcache_max = 1024
EOF
        log "创建jemalloc配置文件: $JEMALLOC_CONF"
    fi
    
    # 为目标进程配置jemalloc
    for proc in "${TARGET_PROCESSES[@]}"; do
        local service_file="/etc/systemd/system/$proc.service"
        if [ -f "$service_file" ]; then
            if ! grep -q "LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so" "$service_file"; then
                sed -i '/ExecStart=/i Environment="LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so"' "$service_file"
                systemctl daemon-reload
                log "为$proc配置jemalloc内存分配器"
            fi
        fi
    done
    
    return 0
}

# 调整内核内存管理参数
tune_kernel_memory_params() {
    log "调整内核内存管理参数"
    
    # 调整内存碎片整理参数
    sysctl -w vm.extfrag_threshold=500
    sysctl -w vm.zone_reclaim_mode=0
    sysctl -w vm.dirty_background_ratio=5
    sysctl -w vm.dirty_ratio=10
    
    # 保存配置
    cat >> /etc/sysctl.conf << EOF
vm.extfrag_threshold=500
vm.zone_reclaim_mode=0
vm.dirty_background_ratio=5
vm.dirty_ratio=10
EOF
    
    log "内核内存参数调整完成"
}

# 主逻辑
main() {
    log "开始内存分配器碎片检查"
    
    # 检查系统级内存碎片
    local system_frag=$(check_system_fragmentation)
    log "系统内存碎片率: $system_frag%"
    
    # 检测是否需要处理
    if [ $system_frag -lt $FRAGMENTATION_THRESHOLD ]; then
        log "系统内存碎片率在正常范围内"
        exit 0
    fi
    
    log "检测到系统内存碎片率过高，启动处理流程"
    
    # 步骤1: 调整内核参数
    tune_kernel_memory_params
    
    # 步骤2: 配置优化的内存分配器
    configure_jemalloc
    
    # 步骤3: 检查并重启高碎片率进程
    local high_frag_processes=""
    for proc in "${TARGET_PROCESSES[@]}"; do
        high_frag_processes+=$(check_process_fragmentation "$proc")
    done
    
    if [ -n "$high_frag_processes" ]; then
        log "发现高碎片率进程，准备重启"
        echo "$high_frag_processes" | sort -k3nr | while read -r pid proc ratio; do
            restart_process "$pid" "$proc"
        done
    else
        log "未发现需要重启的高碎片率进程"
    fi
    
    # 检查优化效果
    sleep 120  # 等待优化生效
    local new_frag=$(check_system_fragmentation)
    log "优化后系统内存碎片率: $new_frag%"
    
    if [ $new_frag -lt $FRAGMENTATION_THRESHOLD ]; then
        log "内存碎片优化成功"
        exit 0
    else
        log "内存碎片优化效果不明显，建议人工干预"
        exit 1
    fi
}

# 执行主逻辑
main
    